{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch as t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy + L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 300)\n",
    "        self.fc2 = nn.Linear(300, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist_loader import load_data_shared, vectorized_result\n",
    "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n",
    "                                                                     seed=666,\n",
    "                                                                     train_size=1000,\n",
    "                                                                     vali_size=0,\n",
    "                                                                     test_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(data, net):\n",
    "    with t.no_grad():\n",
    "        #for index in range(test_data[0].shape[0]):\n",
    "            # get the inputs\n",
    "        inputs, labels = t.Tensor(data[0]), t.Tensor(data[1])\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = t.max(outputs, 1)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "\n",
    "        correct = (predicted == labels).sum().item()\n",
    "        accuracy = correct / data[0].shape[0]\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(net, criterion, optimizer, is_early_stopping):\n",
    "    loss_scores = []\n",
    "    validate_loss = []\n",
    "    \n",
    "    # initialize the early_stopping object\n",
    "    early_stopping = EarlyStopping(patience=20, verbose=True)\n",
    "    \n",
    "    for epoch in range(1000):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        #train_scores.append(predict(training_data, net))\n",
    "        one_vali_loss = predict(test_data, net)\n",
    "        validate_loss.append(one_vali_loss)\n",
    "        \n",
    "        if is_early_stopping:\n",
    "            early_stopping(one_vali_loss, net)\n",
    "\n",
    "            if early_stopping.early_stop:\n",
    "                print(\"Early stopping\")\n",
    "                break\n",
    "                \n",
    "    print('Finished Training')\n",
    "    return loss_scores, validate_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net = Net()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr = 1e-1, weight_decay=1e-2)\n",
    "loss_scores, validate_loss = fit(net, criterion, optimizer, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAfUElEQVR4nO3deXRc5Z3m8e+vFlVplyzJkmzZlm1kg81qHGO2E5olAUJCFpIOmawNzUwO3ZP0JD0Jme6kkzN9Mlv2EAiTEJo0TTJJ6DQhCzTEhABhkc1mY4x3LFuWJVn7Vts7f9SVLQvZ2kq+rqrnc8499973vlX1u77yc6tuvVVlzjlERCT7BfwuQEREMkOBLiKSIxToIiI5QoEuIpIjFOgiIjki5NcDV1dXu8bGRr8eXkQkK23cuLHDOVcz0TbfAr2xsZHm5ma/Hl5EJCuZ2d7jbdMlFxGRHKFAFxHJEQp0EZEcoUAXEckRCnQRkRyhQBcRyREKdBGRHJF1gT7c+Qbb7rkVl4j5XYqIyCkl6wK9+enfs3LPP7P1Z//gdykiIqeUrAv0i97xMZ6IXEbTtjvp2qVPmoqIjMq6QA8EjIUf+g7droSBn94CuvQiIgJkYaADLF+ymD+t+nsaRnay+1+/7Hc5IiKnhKwMdICr33cTj4YvY9GWOxjau8nvckREfJe1gV4QClD9/m9w2JXS+5ObdelFRPJe1gY6wLkrlvHwss9TO7STtoe+4nc5IiK+yupAB7j+z2/m14HLqH7xduItuvQiIvkr6wO9LBomct3/ot2V03f/zZAY8bskERFfZH2gA1y5ZiU/rfsM8wZ2MvDH7/ldjoiIL3Ii0AGuu+ET/CF1Dvbk12Coy+9yREROupwJ9OU1JWxe9V+IJvrpevh/+F2OiMhJlzOBDvDB667hIS6l+KUfwUCH3+WIiJxUORXoVSUR2s69lVAqRtfvv+V3OSIiJ1VOBTrAe676Mx5hHdEX7obhHr/LERE5aXIu0KtLIuw74z9RmOqn66m7/S5HROSkyblAB7jummvYlGrCPfdDSKX8LkdE5KTIyUCvLy/kpfobmDeyj9iODX6XIyJyUuRkoAOcfvlHOOxKaP+9PmgkIvkhZwN9/YoFPFpwBfMPPg6Dh/0uR0RkzuVsoJsZwbPfT5gEnc0/97scEZE5l7OBDrD+kivYnaplcOP/87sUEZE5l9OBvrCyiI2lV7CwpxnX2+p3OSIicyqnAx0gct77CeBofVaXXUQkt+V8oF90wUXscbUMv/o7v0sREZlTkwa6mS0ysw1mttXMtpjZpyboY2b2bTPbYWYvm9mauSl3+qpKo2wpuoCFXc9DfMjvckRE5sxUnqEngM84584A1gO3mtmqcX2uAZq86RbgjoxWOUvJ5VcSYYTuV/UhIxHJXZMGunOu1Tm3yVvuA7YCC8d1ux6416U9A1SYWX3Gq52hpnXXMOQKOPTCQ36XIiIyZ6Z1Dd3MGoHzgGfHbVoI7Buz3sKbQ983py+q4ZXASopan/G7FBGROTPlQDezEuAXwKedc73jN09wEzfBfdxiZs1m1tze3j69SmfBzGivegsLRnbhBvXzdCKSm6YU6GYWJh3m9znnHpigSwuwaMx6A3BgfCfn3F3OubXOubU1NTUzqXfGIssvIYDj4GZdRxeR3DSVUS4G/BDY6pz7+nG6PQh81Bvtsh7occ6dUp/kWX7uWxlxIbq3Pu53KSIicyI0hT4XAx8BXjGzF722LwCLAZxzdwK/Aa4FdgCDwCcyX+rsNNZV8WKgicrW5/wuRURkTkwa6M65J5n4GvnYPg64NVNFzQUzo7PibFZ1PQCJGIQK/C5JRCSjcv6TosdYcC4R4nTvfdnvSkREMi6vAr1qxXoADr72J58rERHJvLwK9KaVZ9Hjiojv2+h3KSIiGZdXgV4SDbMj1ETp4c1+lyIiknF5FegA3RWraYjtwiVG/C5FRCSj8i7Qg3WrCZGka99Wv0sREcmovAv0iiVnA9C+88VJeoqIZJe8C/SGFeeQdMbg/i1+lyIiklF5F+jVFeW0WB3hzm1+lyIiklF5F+gAbdGlVAzs9LsMEZGMystAHypvoi5xABcf9rsUEZGMyctAD9adQchSHNqt8egikjvyMtArFp0JQMdevTEqIrkjLwO9bmn6N66H23b4XImISObkZaBXzZtHhyuHrl1+lyIikjF5GehmRlt4IUX9b/hdiohIxuRloAP0FS2iOtbidxkiIhmTt4GeKG+kxh0mNtjndykiIhmRt4EermkC4ODe13yuREQkM/I20MsaVgLoWxdFJGfkbaDXNaaHLo4c0tBFEckNeRvoFZVV9Lhi6N7ndykiIhmRt4FuZrSHaokOaKSLiOSGvA10gN5IPeUjrX6XISKSEXkd6CMlDdQkD+FSKb9LERGZtbwOdKtYTJGN0NPZ5ncpIiKzlteBXlDdCEB7y3Z/CxERyYC8DvTyumUA9B3UrxeJSPbL60CvXrQCgJGOvT5XIiIye3kd6OWV1fRShPXoWxdFJPvldaADdARriQ7s97sMEZFZy/tA11h0EckVeR/oIyUNzE+2aSy6iGS9vA90q1hEsQ1zuPOQ36WIiMxK3gd6pHopAO37NBZdRLJb3gd6+YLTAI1FF5Hsl/eBXt2QDvRYp8aii0h2y/tALymvpp9CjUUXkaw3aaCb2d1mdsjMNh9n+2Vm1mNmL3rTFzNf5hwyoyM4X2PRRSTrhabQ5x7gu8C9J+jzR+fcdRmpyAe9kXrKhg/6XYaIyKxM+gzdOfcEcPgk1OKbWMnC9Fh05/wuRURkxjJ1Df1CM3vJzH5rZquP18nMbjGzZjNrbm9vz9BDZ0DFEspskM6OU6gmEZFpykSgbwKWOOfOAb4D/PJ4HZ1zdznn1jrn1tbU1GTgoTMj4n0veoe+F11EstisA9051+uc6/eWfwOEzax61pWdRGXe96L3tu32uRIRkZmbdaCbWZ2Zmbe8zrvPztne78lUtagJgFjHHn8LERGZhUlHuZjZ/cBlQLWZtQBfAsIAzrk7gRuAT5pZAhgCPuiy7N3FkopahiiAbo1FF5HsNWmgO+dunGT7d0kPa8xeZrQHa4loLLqIZLG8/6ToqD59L7qIZDkFume4eCE1yUMaiy4iWUuBPqpiEZXWR8fhnP4MlYjkMAW6Z3QsenvLDn8LERGZIQW6p7R2OQC9+l50EclSCnRP9SLve9E1Fl1EspQC3VNcuYARwtC9z+9SRERmRIE+KhCgI1BDZKDF70pERGZEgT5Gb7SesmGNRReR7KRAH0Nj0UUkmynQxypfTLX10N6lsegikn0U6GOE56eHLna+sc3nSkREpk+BPkbpgtMB6Dvwus+ViIhMnwJ9jNolZwAwckifFhWR7KNAHyNaWkkXZQS79MtFIpJ9FOjjtBc0UDqoH7oQkeyjQB+nv3gxNXH90IWIZB8F+jipyqXU0UlXd4/fpYiITIsCfZxobfoHo1v3bvW5EhGR6VGgj1PRkB662LNPY9FFJLso0MeZv2QVALF2DV0UkeyiQB+noKSSbsoIdu/yuxQRkWlRoE+go2AhpQMauigi2UWBPoGBkkZq4y2kUvrWRRHJHgr0CbialdTZYQ60tfldiojIlCnQJ1DSsBqA1h0v+VyJiMjUKdAnMP+08wDob9nscyUiIlOnQJ9AWe1yhinA2l/zuxQRkSlToE8kEKA1vJjSPo1FF5HsoUA/jr7S06iP7dVIFxHJGgr046lZyQLrpEUjXUQkSyjQj6Oo4UwAWne87HMlIiJTo0A/jvomb6TLGxq6KCLZQYF+HMXzlzNAIYG2V/wuRURkShToxxMI0FrYRHWfhi6KSHZQoJ/AUNVqlqf2cLhvyO9SREQmpUA/geii8yiyEXZte9HvUkREJjVpoJvZ3WZ2yMwm/By8pX3bzHaY2ctmtibzZfqj7vT1AHTv2uhzJSIik5vKM/R7gKtPsP0aoMmbbgHumH1Zp4bShlWMEMZaNdJFRE59kwa6c+4J4PAJulwP3OvSngEqzKw+UwX6KhimNbKMyh79YLSInPoycQ19IbBvzHqL1/YmZnaLmTWbWXN7e3sGHnruDVSdyWnJnXTqjVEROcVlItBtgrYJvwDFOXeXc26tc25tTU1NBh567hUtu5AyG+S1V5r9LkVE5IQyEegtwKIx6w3AgQzc7ylhwVlvBaBn+5M+VyIicmKZCPQHgY96o13WAz3OudYM3O8pITK/iR4rI9qqkS4icmoLTdbBzO4HLgOqzawF+BIQBnDO3Qn8BrgW2AEMAp+Yq2J9YUZb+Tk0dm1mJJEkEgr6XZGIyIQmDXTn3I2TbHfArRmr6BTkGtaxrPuPvLhzD+euXO53OSIiE9InRadg/ur0dfTWV/7gcyUiIsenQJ+CytMuIEYY9vzR71JERI5LgT4V4SgtpeewrK+Z4XjS72pERCakQJ8it/StrLQ3ePm11/0uRURkQgr0Kapfcy0AB196xOdKREQmpkCfoqLF59FnpRTu03V0ETk1KdCnKhDkYNU6Vg9v5FCPvtdFRE49CvRpKDnzGhbYYTY++4TfpYiIvIkCfRrq1r6bJAFiWx70uxQRkTdRoE+DldTQUnI2K7ufoH8k4Xc5IiLHUKBPkzv9HZxub/DcJn1Zl4icWhTo09Sw/v0AHG5+wOdKRESOpUCfplD1UvYXncGqjt/RMxj3uxwRkSMU6DNxzgdZZXt46ml9WZeInDoU6DOw4JIPkyBI/IX7/S5FROQIBfoMWHE1++ZdxAX9j7GrrdvvckREAAX6jFVeejN11sXGR+7zuxQREUCBPmMV57yTjlAdS3bex1BMX6krIv5ToM9UIMjg2R9nHVv4wx83+F2NiIgCfTYWXXELw0TgT7eTTDm/yxGRPKdAnwUrruLAaX/OlfHHeeK5Zr/LEZE8p0CfpSXvvI2UBRnZ8L9xTs/SRcQ/CvRZCpYvYO+SG7h8+FH+9PxzfpcjInlMgZ4Bje/9EgkLwyN/RyKZ8rscEclTCvQMCJfXs+/MW7ko8Rx/+N3P/C5HRPKUAj1DVlz/t7QF62h8/iv09ff7XY6I5CEFeoZYuJDBK7/KclrYdO/n/S5HRPKQAj2Dll74Xl6oegeXtP0zW5//vd/liEieUaBn2IqPfpcOm0fZbz7JUE+n3+WISB5RoGdYcfk8Wt92BzWpdvb84MOQ0qgXETk5FOhz4NyL3s7jjX/DGX1Ps/0nn/O7HBHJEwr0OfJnH/kCD0evoen1u2h/9Nt+lyMieUCBPkfCoSCr//IuNtg6qp78Iv3P/MjvkkQkxynQ51BDVRnlH76Xp1JnUfK7T5N4+na/SxKRHKZAn2NrltfT8+4f89vkWwg98gWSj3wRUvpBDBHJPAX6SXDdmkY6r76T+xJXEHz6W6T+5QMwpN8iFZHMUqCfJB+++DSGr/4aX4jfhNuxAXfHxbD7Cb/LEpEcMqVAN7OrzWybme0wszd9rt3MPm5m7Wb2ojfdnPlSs99Nlyxl2dV/xftGvkTrgIN/eif89nMw3Ot3aSKSAyYNdDMLArcD1wCrgBvNbNUEXX/qnDvXm36Q4Tpzxs2XLuPjH7iBtw//I/9WcB08eyd853zY9GN9CElEZmUqz9DXATucc7ucczHgJ8D1c1tWbnv3eQu54+OX8vexj/IhvkpP4UJ48K/g+5fCln9VsIvIjEwl0BcC+8ast3ht473PzF42s5+b2aKMVJfDLmmq5ld/fQldlWdx7v7P8usV/x0XH4affRy+tx423gOxAb/LFJEsMpVAtwnaxv945q+ARufc2cCjwD9NeEdmt5hZs5k1t7e3T6/SHLSkqpgHPnkR7z1vEbe+vIz32tc5eNX3IFgAv/oUfO10+M3fwoEXQb9XKiKTsMl+2NjMLgT+wTn3dm/9NgDn3FeP0z8IHHbOlZ/ofteuXeuam5tnVHQu+vXLrfzdL19hIJbks1c18RdLDhHadE/6EkwyBvOWwer3wqrroe4ssInOsyKS68xso3Nu7YTbphDoIeB14ApgP/A88CHn3JYxfeqdc63e8nuAzznn1p/ofhXob9beN8JtD7zCo1vbWFlbyj+8azUX1hts/RVseSA9zNGloKQOll8Op10Byy6D4mq/SxeRk2RWge7dwbXAN4EgcLdz7h/N7CtAs3PuQTP7KvAuIAEcBj7pnHvtRPepQJ+Yc45/f7WNrzz0Ki1dQ7zj7Ho++7aVLK0uhv522P4w7HgMdm2Aoa70jeYth0Xr0lPDOqg5HYIhf3dERObErAN9LijQT2w4nuR7j+/k/z6xi1gyxQfWNvDXlzexoKIw3SGVhAMvwJ4nYd9z0PIcDHjvSwQjULMSas+E2tVQuwqqV0JpPQT0WTKRbKZAz2KH+ob53oad3PfsXsyM961p4C8vXcqympJjOzoHXbth3/PQ9gq0vQptW6D/4NE+oShULk1fj5+3ND2VL4ayeihbCIWVujYvcopToOeAlq5Bbt+wk19saiGeTHHVGbX8x7cuY83iSuxEITzQkQ72zu1weDcc3pWeuvZAYvjYvqFo+ll82cJ0yJfWp6/PF1VDUZW37M0LShT+Ij5QoOeQ9r4R7v3THn78zF66B+Osqi/jxgsWc/25CyiLhqd+R6kU9LVC737oPZCe+rx5r9fedxCSIxPfPhhJh3u03JvK0vNI2bjl8qPLkRIIF0FBcXoeLtIlIJFpUqDnoMFYggc27edfnn2DV1t7KQwHue7set53fgPrGucRCGTg2bNzEOtPP8sf7PTmHUfng4dhuCc9jfR6y73p5VRiao8RKkwHfEERhEfno6FfmH7VECxIz0Oj80j6hDJh2+g0UZvXHozoRCJZS4Gew5xzvLK/h/uf28eDL+5nIJakrizKO86u513nLODshvITX5KZm6IgPng03EeDPtafbo8NePPBMW2DEB/w5l6fxDAkRrxpOD0ePz7Emz/XNgOB8LiQD0MgNGYKznJ9lrexYIbvVyewXKFAzxODsQSPbj3Er146wB+2tRNLplg8r4i3r67l8tNrWdtYSTiY5f+xnUs/+z8m6EfGrI8J/4lOCMe0jb19DFwyfd+p0XniOOtT6TNu3Xc2QegH0ycOC4ybbIK2SbYHgrO7/ZS3B6dw+0zUEEifBGd7H8frU1QFJfNndiQV6PmnZyjOw1sO8tDLrTyzs5NYMkVZNMRbV87nyjPmc2lTDfOKC/wuMz84l/5A2KxPEqNtyVnez+h6/GhtLjVu+XjT+D7JWd5+ittTyclvm4lXbifLxZ+Gq748o5sq0PNc/0iCJ7e389jWQ2zYdoiO/hgAq+rLuPi0Ki5aXs26pfMojujDSJLFnMvQSWUq0ywfp6oJ6s6c0W4q0OWIVMrxUks3T27v4KmdHWza200smSIUMM5dVMFbls7j/MWVrFlSqWfwIqcgBboc13A8SfOeLp7a2cHTOzvZsr+HRCr9N7G0upg1iys5f0kla5ZUcFpNCaFsvwYvkuVOFOh6jZ3nouEglzRVc0lT+gu+huNJXm7pYdMbXWzc28Xj2w7xi00tAERCAc6oL2P1gjLOXFjOmQvKWVFXQiQU9HMXRMSjZ+hyQs459nYO8sK+Lrbs72XzgR62HOilbzg9ciMUMFbUlnJ6fSkraktZUVtC0/xSFlYUZmYsvIgcQ8/QZcbMjMbqYhqri3nPeek25xz7Dg+x+UAPm/f3sPlAL0/t6OCBTfuP3K6oIMhp89PhvqK2hKbaEpZWl9BQWZj9QydFTlF6hi4Z0zMYZ/uhPrYf6uf1tj62t6Xnh/qOfn1AMGA0VBbSWFXM0upiGquKWFJdzNKqYhoqC3WNXmQSeoYuJ0V5UZi1jfNY2zjvmPaewTg72vvY3THIno4BdncOsKdjgOY9hxmIJY/0CwWMhZWFNFQWsrCikIUVRellb72+PKrAFzkBBbrMufKiMOcvmcf5S44NeuccHf0x9nQOsLtjgL2dA+ztHGR/9xAbtrXT3nfsF4MFA0ZdWTQd9pWFLKiIUlcWpdab6sqjVJdECOraveQpBbr4xsyoKY1QUxrhLeOe1UN6xE1rzzD7u4Zo6UoH/f6uIVq6h3hu92EO9g6TTB17yTBgUFMaoa4syvyy0cCPHAn9mtIIVSUFzCsq0LN9yTkKdDllRcNBllanr7VPJJlydA6M0NYzwsHeYdrGTAd7R9h3eJDn9xymezD+ptuaQWVRAVXFBVSXpEO+uiRCdcnoerqtxpsXFei/ipz69FcqWSsYMOaXRplfGuUsyo/bbzie5FBvOvQ7+0fo6B+hoz9GR/8Ind58y4FeOvpG6BuZ+Iu0IqEAlUUFVBSFqSgKe8sF3nKYiqICKosKvOX0ekVhWK8C5KRSoEvOi4aDLK4qYnFV0aR9h+NJOgdidHph3+6dALoH43QNxOgeitM9GGP7oX66B2N0D8aPfLJ2IqXREBVFYcoLw5RF01NpNERZoTefYL0sGqasMERJJKQTgkyLAl1kjGg46I2wKZxSf+cc/SOJdOB7AT9+3j0Yo3c4Qe9QnF0d/fR5y2NH+BxPcUHwmLAviYYojoQoKUjPiyNBbx6iJBKk+Ei7t+4tFxeE9GZxHlCgi8yCmVEaDVMaDbNo3uSvAMZKJFP0jyToHUrQOxyndzh+JOx7hxP0DcfpHfLm3rbO/hhvdA7SP5JgMJZkIJZgqh8liYYDlIwJ+CMng4IQ0XCQwoIARaPL4SBFBel5tCBIUThIYYE3jdke9db1YbFTgwJdxCehYMC7Dj/zb7VMpRxD8SQDI4kjId8/kjhmfXQ5PU8yGDu6vbM/xr7YIMPxFEPx9LbheGradYSDdsyJIOqdAIoKgkRCQaLhAJFQkEgoQCQUIBr2lsfPQ4Fj+4fH9ffaoqEg4aCd/F/jOsUp0EWyWCBgRy6rzOz3b94slXKMJFIMxhIMxZMMx5MMxpIMxZIMxpMMx5Je+Ke3jbYPxcb0HXO7nqE4I/EUw4kkI/EUI4kUI4nkjE4cY5lxNOTHhH5BKEA4GKAgOLps6fVQui0cPNonHDIib2oLpNtCduR+wqGx95e+z8iR5aPto21+Xd5SoIvIMQIBO3J5ZS4554glvYCPHw35kUTySNvRk8Bomzf3lofHtQ3Hk8S9+4wn0686eobSy7FkipjXHk86YomjbZlmRjrsA0Y4FCAUCFAQNELeyeDGdYu5+dJlGX9cBbqI+MLMvGfXQYj6V4dzjkTKpYM+4RhJJo8EftwL/FgyRTzhnQiSSWIJd2Rb+gQxehJJ3y6RSi/HkykSyRSxpCPh9YunHNUlkTnZFwW6iOQ1MztyWYYCgLDfJc2Y3poWEckRCnQRkRyhQBcRyREKdBGRHKFAFxHJEQp0EZEcoUAXEckRCnQRkRxhbqpf1ZbpBzZrB/bO8ObVQEcGy8kG2uf8oH3OD7PZ5yXOuZqJNvgW6LNhZs3OubV+13EyaZ/zg/Y5P8zVPuuSi4hIjlCgi4jkiGwN9Lv8LsAH2uf8oH3OD3Oyz1l5DV1ERN4sW5+hi4jIOAp0EZEckXWBbmZXm9k2M9thZp/3u55MMbNFZrbBzLaa2RYz+5TXPs/M/t3MtnvzSq/dzOzb3r/Dy2a2xt89mBkzC5rZC2b2kLe+1Mye9fb3p2ZW4LVHvPUd3vZGP+ueDTOrMLOfm9lr3vG+MJePs5n9jfc3vdnM7jezaC4eZzO728wOmdnmMW3TPq5m9jGv/3Yz+9h0asiqQDezIHA7cA2wCrjRzFb5W1XGJIDPOOfOANYDt3r79nngMedcE/CYtw7pf4Mmb7oFuOPkl5wRnwK2jln/n8A3vP3tAm7y2m8CupxzpwHf8Pplq28Bv3POnQ6cQ3r/c/I4m9lC4D8Da51zZwJB4IPk5nG+B7h6XNu0jquZzQO+BFwArAO+NHoSmBLnXNZMwIXAw2PWbwNu87uuOdrXfwOuArYB9V5bPbDNW/4+cOOY/kf6ZcsENHh/5JcDDwFG+tNzofHHG3gYuNBbDnn9zO99mME+lwG7x9eeq8cZWAjsA+Z5x+0h4O25epyBRmDzTI8rcCPw/THtx/SbbMqqZ+gc/eMY1eK15RTvZeZ5wLNArXOuFcCbz/e65cK/xTeB/wqM/ux6FdDtnEt462P36cj+ett7vP7ZZhnQDvzIu9T0AzMrJkePs3NuP/B/gDeAVtLHbSO5f5xHTfe4zup4Z1ug2wRtOTXu0sxKgF8An3bO9Z6o6wRtWfNvYWbXAYeccxvHNk/Q1U1hWzYJAWuAO5xz5wEDHH0ZPpGs3m/vcsH1wFJgAVBM+nLDeLl2nCdzvP2c1f5nW6C3AIvGrDcAB3yqJePMLEw6zO9zzj3gNbeZWb23vR445LVn+7/FxcC7zGwP8BPSl12+CVSYWcjrM3afjuyvt70cOHwyC86QFqDFOfest/5z0gGfq8f5SmC3c67dORcHHgAuIveP86jpHtdZHe9sC/TngSbvHfIC0m+uPOhzTRlhZgb8ENjqnPv6mE0PAqPvdH+M9LX10faPeu+Wrwd6Rl/aZQPn3G3OuQbnXCPp4/h759x/ADYAN3jdxu/v6L/DDV7/rHvm5pw7COwzs5Ve0xXAq+TocSZ9qWW9mRV5f+Oj+5vTx3mM6R7Xh4G3mVml9+rmbV7b1Pj9JsIM3nS4Fngd2An8N7/ryeB+XUL6pdXLwIvedC3p64ePAdu9+Tyvv5Ee8bMTeIX0KALf92OG+34Z8JC3vAx4DtgB/AyIeO1Rb32Ht32Z33XPYn/PBZq9Y/1LoDKXjzPwZeA1YDPwYyCSi8cZuJ/0+wRx0s+0b5rJcQX+wtv/HcAnplODPvovIpIjsu2Si4iIHIcCXUQkRyjQRURyhAJdRCRHKNBFRHKEAl1EJEco0EVEcsT/B8FEaY8tkEoXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores)\n",
    "plt.plot(validate_loss)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy + L2 + early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import EarlyStopping\n",
    "from early_stopping import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (inf --> 2.297760).  Saving model ...\n",
      "Validation loss decreased (2.297760 --> 2.293224).  Saving model ...\n",
      "Validation loss decreased (2.293224 --> 2.288750).  Saving model ...\n",
      "Validation loss decreased (2.288750 --> 2.284319).  Saving model ...\n",
      "Validation loss decreased (2.284319 --> 2.279920).  Saving model ...\n",
      "Validation loss decreased (2.279920 --> 2.275534).  Saving model ...\n",
      "Validation loss decreased (2.275534 --> 2.271152).  Saving model ...\n",
      "Validation loss decreased (2.271152 --> 2.266778).  Saving model ...\n",
      "Validation loss decreased (2.266778 --> 2.262383).  Saving model ...\n",
      "Validation loss decreased (2.262383 --> 2.257947).  Saving model ...\n",
      "Validation loss decreased (2.257947 --> 2.253465).  Saving model ...\n",
      "Validation loss decreased (2.253465 --> 2.248921).  Saving model ...\n",
      "Validation loss decreased (2.248921 --> 2.244309).  Saving model ...\n",
      "Validation loss decreased (2.244309 --> 2.239632).  Saving model ...\n",
      "Validation loss decreased (2.239632 --> 2.234889).  Saving model ...\n",
      "Validation loss decreased (2.234889 --> 2.230062).  Saving model ...\n",
      "Validation loss decreased (2.230062 --> 2.225136).  Saving model ...\n",
      "Validation loss decreased (2.225136 --> 2.220093).  Saving model ...\n",
      "Validation loss decreased (2.220093 --> 2.214937).  Saving model ...\n",
      "Validation loss decreased (2.214937 --> 2.209666).  Saving model ...\n",
      "Validation loss decreased (2.209666 --> 2.204286).  Saving model ...\n",
      "Validation loss decreased (2.204286 --> 2.198766).  Saving model ...\n",
      "Validation loss decreased (2.198766 --> 2.193099).  Saving model ...\n",
      "Validation loss decreased (2.193099 --> 2.187288).  Saving model ...\n",
      "Validation loss decreased (2.187288 --> 2.181315).  Saving model ...\n",
      "Validation loss decreased (2.181315 --> 2.175181).  Saving model ...\n",
      "Validation loss decreased (2.175181 --> 2.168855).  Saving model ...\n",
      "Validation loss decreased (2.168855 --> 2.162357).  Saving model ...\n",
      "Validation loss decreased (2.162357 --> 2.155670).  Saving model ...\n",
      "Validation loss decreased (2.155670 --> 2.148775).  Saving model ...\n",
      "Validation loss decreased (2.148775 --> 2.141680).  Saving model ...\n",
      "Validation loss decreased (2.141680 --> 2.134367).  Saving model ...\n",
      "Validation loss decreased (2.134367 --> 2.126817).  Saving model ...\n",
      "Validation loss decreased (2.126817 --> 2.119024).  Saving model ...\n",
      "Validation loss decreased (2.119024 --> 2.110952).  Saving model ...\n",
      "Validation loss decreased (2.110952 --> 2.102597).  Saving model ...\n",
      "Validation loss decreased (2.102597 --> 2.093935).  Saving model ...\n",
      "Validation loss decreased (2.093935 --> 2.084959).  Saving model ...\n",
      "Validation loss decreased (2.084959 --> 2.075655).  Saving model ...\n",
      "Validation loss decreased (2.075655 --> 2.066004).  Saving model ...\n",
      "Validation loss decreased (2.066004 --> 2.055983).  Saving model ...\n",
      "Validation loss decreased (2.055983 --> 2.045565).  Saving model ...\n",
      "Validation loss decreased (2.045565 --> 2.034733).  Saving model ...\n",
      "Validation loss decreased (2.034733 --> 2.023481).  Saving model ...\n",
      "Validation loss decreased (2.023481 --> 2.011774).  Saving model ...\n",
      "Validation loss decreased (2.011774 --> 1.999578).  Saving model ...\n",
      "Validation loss decreased (1.999578 --> 1.986874).  Saving model ...\n",
      "Validation loss decreased (1.986874 --> 1.973619).  Saving model ...\n",
      "Validation loss decreased (1.973619 --> 1.959794).  Saving model ...\n",
      "Validation loss decreased (1.959794 --> 1.945365).  Saving model ...\n",
      "Validation loss decreased (1.945365 --> 1.930281).  Saving model ...\n",
      "Validation loss decreased (1.930281 --> 1.914511).  Saving model ...\n",
      "Validation loss decreased (1.914511 --> 1.898015).  Saving model ...\n",
      "Validation loss decreased (1.898015 --> 1.880757).  Saving model ...\n",
      "Validation loss decreased (1.880757 --> 1.862692).  Saving model ...\n",
      "Validation loss decreased (1.862692 --> 1.843802).  Saving model ...\n",
      "Validation loss decreased (1.843802 --> 1.824051).  Saving model ...\n",
      "Validation loss decreased (1.824051 --> 1.803435).  Saving model ...\n",
      "Validation loss decreased (1.803435 --> 1.781944).  Saving model ...\n",
      "Validation loss decreased (1.781944 --> 1.759575).  Saving model ...\n",
      "Validation loss decreased (1.759575 --> 1.736351).  Saving model ...\n",
      "Validation loss decreased (1.736351 --> 1.712331).  Saving model ...\n",
      "Validation loss decreased (1.712331 --> 1.687576).  Saving model ...\n",
      "Validation loss decreased (1.687576 --> 1.662166).  Saving model ...\n",
      "Validation loss decreased (1.662166 --> 1.636209).  Saving model ...\n",
      "Validation loss decreased (1.636209 --> 1.609815).  Saving model ...\n",
      "Validation loss decreased (1.609815 --> 1.583103).  Saving model ...\n",
      "Validation loss decreased (1.583103 --> 1.556180).  Saving model ...\n",
      "Validation loss decreased (1.556180 --> 1.529158).  Saving model ...\n",
      "Validation loss decreased (1.529158 --> 1.502137).  Saving model ...\n",
      "Validation loss decreased (1.502137 --> 1.475209).  Saving model ...\n",
      "Validation loss decreased (1.475209 --> 1.448464).  Saving model ...\n",
      "Validation loss decreased (1.448464 --> 1.421989).  Saving model ...\n",
      "Validation loss decreased (1.421989 --> 1.395845).  Saving model ...\n",
      "Validation loss decreased (1.395845 --> 1.370100).  Saving model ...\n",
      "Validation loss decreased (1.370100 --> 1.344800).  Saving model ...\n",
      "Validation loss decreased (1.344800 --> 1.319972).  Saving model ...\n",
      "Validation loss decreased (1.319972 --> 1.295655).  Saving model ...\n",
      "Validation loss decreased (1.295655 --> 1.271884).  Saving model ...\n",
      "Validation loss decreased (1.271884 --> 1.248676).  Saving model ...\n",
      "Validation loss decreased (1.248676 --> 1.226047).  Saving model ...\n",
      "Validation loss decreased (1.226047 --> 1.203989).  Saving model ...\n",
      "Validation loss decreased (1.203989 --> 1.182513).  Saving model ...\n",
      "Validation loss decreased (1.182513 --> 1.161643).  Saving model ...\n",
      "Validation loss decreased (1.161643 --> 1.141369).  Saving model ...\n",
      "Validation loss decreased (1.141369 --> 1.121682).  Saving model ...\n",
      "Validation loss decreased (1.121682 --> 1.102573).  Saving model ...\n",
      "Validation loss decreased (1.102573 --> 1.084034).  Saving model ...\n",
      "Validation loss decreased (1.084034 --> 1.066051).  Saving model ...\n",
      "Validation loss decreased (1.066051 --> 1.048618).  Saving model ...\n",
      "Validation loss decreased (1.048618 --> 1.031713).  Saving model ...\n",
      "Validation loss decreased (1.031713 --> 1.015331).  Saving model ...\n",
      "Validation loss decreased (1.015331 --> 0.999457).  Saving model ...\n",
      "Validation loss decreased (0.999457 --> 0.984074).  Saving model ...\n",
      "Validation loss decreased (0.984074 --> 0.969163).  Saving model ...\n",
      "Validation loss decreased (0.969163 --> 0.954724).  Saving model ...\n",
      "Validation loss decreased (0.954724 --> 0.940736).  Saving model ...\n",
      "Validation loss decreased (0.940736 --> 0.927190).  Saving model ...\n",
      "Validation loss decreased (0.927190 --> 0.914077).  Saving model ...\n",
      "Validation loss decreased (0.914077 --> 0.901386).  Saving model ...\n",
      "Validation loss decreased (0.901386 --> 0.889085).  Saving model ...\n",
      "Validation loss decreased (0.889085 --> 0.877172).  Saving model ...\n",
      "Validation loss decreased (0.877172 --> 0.865630).  Saving model ...\n",
      "Validation loss decreased (0.865630 --> 0.854456).  Saving model ...\n",
      "Validation loss decreased (0.854456 --> 0.843631).  Saving model ...\n",
      "Validation loss decreased (0.843631 --> 0.833146).  Saving model ...\n",
      "Validation loss decreased (0.833146 --> 0.822993).  Saving model ...\n",
      "Validation loss decreased (0.822993 --> 0.813153).  Saving model ...\n",
      "Validation loss decreased (0.813153 --> 0.803619).  Saving model ...\n",
      "Validation loss decreased (0.803619 --> 0.794379).  Saving model ...\n",
      "Validation loss decreased (0.794379 --> 0.785418).  Saving model ...\n",
      "Validation loss decreased (0.785418 --> 0.776731).  Saving model ...\n",
      "Validation loss decreased (0.776731 --> 0.768306).  Saving model ...\n",
      "Validation loss decreased (0.768306 --> 0.760137).  Saving model ...\n",
      "Validation loss decreased (0.760137 --> 0.752208).  Saving model ...\n",
      "Validation loss decreased (0.752208 --> 0.744515).  Saving model ...\n",
      "Validation loss decreased (0.744515 --> 0.737047).  Saving model ...\n",
      "Validation loss decreased (0.737047 --> 0.729800).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.729800 --> 0.722754).  Saving model ...\n",
      "Validation loss decreased (0.722754 --> 0.715924).  Saving model ...\n",
      "Validation loss decreased (0.715924 --> 0.709277).  Saving model ...\n",
      "Validation loss decreased (0.709277 --> 0.702824).  Saving model ...\n",
      "Validation loss decreased (0.702824 --> 0.696546).  Saving model ...\n",
      "Validation loss decreased (0.696546 --> 0.690445).  Saving model ...\n",
      "Validation loss decreased (0.690445 --> 0.684509).  Saving model ...\n",
      "Validation loss decreased (0.684509 --> 0.678733).  Saving model ...\n",
      "Validation loss decreased (0.678733 --> 0.673116).  Saving model ...\n",
      "Validation loss decreased (0.673116 --> 0.667654).  Saving model ...\n",
      "Validation loss decreased (0.667654 --> 0.662338).  Saving model ...\n",
      "Validation loss decreased (0.662338 --> 0.657155).  Saving model ...\n",
      "Validation loss decreased (0.657155 --> 0.652115).  Saving model ...\n",
      "Validation loss decreased (0.652115 --> 0.647201).  Saving model ...\n",
      "Validation loss decreased (0.647201 --> 0.642419).  Saving model ...\n",
      "Validation loss decreased (0.642419 --> 0.637759).  Saving model ...\n",
      "Validation loss decreased (0.637759 --> 0.633219).  Saving model ...\n",
      "Validation loss decreased (0.633219 --> 0.628796).  Saving model ...\n",
      "Validation loss decreased (0.628796 --> 0.624483).  Saving model ...\n",
      "Validation loss decreased (0.624483 --> 0.620277).  Saving model ...\n",
      "Validation loss decreased (0.620277 --> 0.616180).  Saving model ...\n",
      "Validation loss decreased (0.616180 --> 0.612175).  Saving model ...\n",
      "Validation loss decreased (0.612175 --> 0.608278).  Saving model ...\n",
      "Validation loss decreased (0.608278 --> 0.604463).  Saving model ...\n",
      "Validation loss decreased (0.604463 --> 0.600753).  Saving model ...\n",
      "Validation loss decreased (0.600753 --> 0.597123).  Saving model ...\n",
      "Validation loss decreased (0.597123 --> 0.593584).  Saving model ...\n",
      "Validation loss decreased (0.593584 --> 0.590123).  Saving model ...\n",
      "Validation loss decreased (0.590123 --> 0.586745).  Saving model ...\n",
      "Validation loss decreased (0.586745 --> 0.583441).  Saving model ...\n",
      "Validation loss decreased (0.583441 --> 0.580216).  Saving model ...\n",
      "Validation loss decreased (0.580216 --> 0.577066).  Saving model ...\n",
      "Validation loss decreased (0.577066 --> 0.573982).  Saving model ...\n",
      "Validation loss decreased (0.573982 --> 0.570974).  Saving model ...\n",
      "Validation loss decreased (0.570974 --> 0.568027).  Saving model ...\n",
      "Validation loss decreased (0.568027 --> 0.565151).  Saving model ...\n",
      "Validation loss decreased (0.565151 --> 0.562330).  Saving model ...\n",
      "Validation loss decreased (0.562330 --> 0.559581).  Saving model ...\n",
      "Validation loss decreased (0.559581 --> 0.556882).  Saving model ...\n",
      "Validation loss decreased (0.556882 --> 0.554249).  Saving model ...\n",
      "Validation loss decreased (0.554249 --> 0.551665).  Saving model ...\n",
      "Validation loss decreased (0.551665 --> 0.549145).  Saving model ...\n",
      "Validation loss decreased (0.549145 --> 0.546672).  Saving model ...\n",
      "Validation loss decreased (0.546672 --> 0.544253).  Saving model ...\n",
      "Validation loss decreased (0.544253 --> 0.541889).  Saving model ...\n",
      "Validation loss decreased (0.541889 --> 0.539569).  Saving model ...\n",
      "Validation loss decreased (0.539569 --> 0.537301).  Saving model ...\n",
      "Validation loss decreased (0.537301 --> 0.535077).  Saving model ...\n",
      "Validation loss decreased (0.535077 --> 0.532894).  Saving model ...\n",
      "Validation loss decreased (0.532894 --> 0.530759).  Saving model ...\n",
      "Validation loss decreased (0.530759 --> 0.528665).  Saving model ...\n",
      "Validation loss decreased (0.528665 --> 0.526621).  Saving model ...\n",
      "Validation loss decreased (0.526621 --> 0.524610).  Saving model ...\n",
      "Validation loss decreased (0.524610 --> 0.522646).  Saving model ...\n",
      "Validation loss decreased (0.522646 --> 0.520719).  Saving model ...\n",
      "Validation loss decreased (0.520719 --> 0.518828).  Saving model ...\n",
      "Validation loss decreased (0.518828 --> 0.516976).  Saving model ...\n",
      "Validation loss decreased (0.516976 --> 0.515158).  Saving model ...\n",
      "Validation loss decreased (0.515158 --> 0.513377).  Saving model ...\n",
      "Validation loss decreased (0.513377 --> 0.511635).  Saving model ...\n",
      "Validation loss decreased (0.511635 --> 0.509916).  Saving model ...\n",
      "Validation loss decreased (0.509916 --> 0.508232).  Saving model ...\n",
      "Validation loss decreased (0.508232 --> 0.506589).  Saving model ...\n",
      "Validation loss decreased (0.506589 --> 0.504961).  Saving model ...\n",
      "Validation loss decreased (0.504961 --> 0.503383).  Saving model ...\n",
      "Validation loss decreased (0.503383 --> 0.501822).  Saving model ...\n",
      "Validation loss decreased (0.501822 --> 0.500290).  Saving model ...\n",
      "Validation loss decreased (0.500290 --> 0.498796).  Saving model ...\n",
      "Validation loss decreased (0.498796 --> 0.497315).  Saving model ...\n",
      "Validation loss decreased (0.497315 --> 0.495878).  Saving model ...\n",
      "Validation loss decreased (0.495878 --> 0.494457).  Saving model ...\n",
      "Validation loss decreased (0.494457 --> 0.493065).  Saving model ...\n",
      "Validation loss decreased (0.493065 --> 0.491699).  Saving model ...\n",
      "Validation loss decreased (0.491699 --> 0.490358).  Saving model ...\n",
      "Validation loss decreased (0.490358 --> 0.489033).  Saving model ...\n",
      "Validation loss decreased (0.489033 --> 0.487739).  Saving model ...\n",
      "Validation loss decreased (0.487739 --> 0.486466).  Saving model ...\n",
      "Validation loss decreased (0.486466 --> 0.485211).  Saving model ...\n",
      "Validation loss decreased (0.485211 --> 0.483983).  Saving model ...\n",
      "Validation loss decreased (0.483983 --> 0.482774).  Saving model ...\n",
      "Validation loss decreased (0.482774 --> 0.481589).  Saving model ...\n",
      "Validation loss decreased (0.481589 --> 0.480418).  Saving model ...\n",
      "Validation loss decreased (0.480418 --> 0.479273).  Saving model ...\n",
      "Validation loss decreased (0.479273 --> 0.478140).  Saving model ...\n",
      "Validation loss decreased (0.478140 --> 0.477031).  Saving model ...\n",
      "Validation loss decreased (0.477031 --> 0.475938).  Saving model ...\n",
      "Validation loss decreased (0.475938 --> 0.474872).  Saving model ...\n",
      "Validation loss decreased (0.474872 --> 0.473807).  Saving model ...\n",
      "Validation loss decreased (0.473807 --> 0.472779).  Saving model ...\n",
      "Validation loss decreased (0.472779 --> 0.471751).  Saving model ...\n",
      "Validation loss decreased (0.471751 --> 0.470752).  Saving model ...\n",
      "Validation loss decreased (0.470752 --> 0.469769).  Saving model ...\n",
      "Validation loss decreased (0.469769 --> 0.468787).  Saving model ...\n",
      "Validation loss decreased (0.468787 --> 0.467843).  Saving model ...\n",
      "Validation loss decreased (0.467843 --> 0.466896).  Saving model ...\n",
      "Validation loss decreased (0.466896 --> 0.465975).  Saving model ...\n",
      "Validation loss decreased (0.465975 --> 0.465061).  Saving model ...\n",
      "Validation loss decreased (0.465061 --> 0.464170).  Saving model ...\n",
      "Validation loss decreased (0.464170 --> 0.463285).  Saving model ...\n",
      "Validation loss decreased (0.463285 --> 0.462422).  Saving model ...\n",
      "Validation loss decreased (0.462422 --> 0.461564).  Saving model ...\n",
      "Validation loss decreased (0.461564 --> 0.460724).  Saving model ...\n",
      "Validation loss decreased (0.460724 --> 0.459894).  Saving model ...\n",
      "Validation loss decreased (0.459894 --> 0.459077).  Saving model ...\n",
      "Validation loss decreased (0.459077 --> 0.458277).  Saving model ...\n",
      "Validation loss decreased (0.458277 --> 0.457480).  Saving model ...\n",
      "Validation loss decreased (0.457480 --> 0.456707).  Saving model ...\n",
      "Validation loss decreased (0.456707 --> 0.455939).  Saving model ...\n",
      "Validation loss decreased (0.455939 --> 0.455181).  Saving model ...\n",
      "Validation loss decreased (0.455181 --> 0.454439).  Saving model ...\n",
      "Validation loss decreased (0.454439 --> 0.453701).  Saving model ...\n",
      "Validation loss decreased (0.453701 --> 0.452988).  Saving model ...\n",
      "Validation loss decreased (0.452988 --> 0.452267).  Saving model ...\n",
      "Validation loss decreased (0.452267 --> 0.451579).  Saving model ...\n",
      "Validation loss decreased (0.451579 --> 0.450874).  Saving model ...\n",
      "Validation loss decreased (0.450874 --> 0.450205).  Saving model ...\n",
      "Validation loss decreased (0.450205 --> 0.449529).  Saving model ...\n",
      "Validation loss decreased (0.449529 --> 0.448872).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.448872 --> 0.448220).  Saving model ...\n",
      "Validation loss decreased (0.448220 --> 0.447579).  Saving model ...\n",
      "Validation loss decreased (0.447579 --> 0.446949).  Saving model ...\n",
      "Validation loss decreased (0.446949 --> 0.446325).  Saving model ...\n",
      "Validation loss decreased (0.446325 --> 0.445707).  Saving model ...\n",
      "Validation loss decreased (0.445707 --> 0.445106).  Saving model ...\n",
      "Validation loss decreased (0.445106 --> 0.444504).  Saving model ...\n",
      "Validation loss decreased (0.444504 --> 0.443921).  Saving model ...\n",
      "Validation loss decreased (0.443921 --> 0.443334).  Saving model ...\n",
      "Validation loss decreased (0.443334 --> 0.442761).  Saving model ...\n",
      "Validation loss decreased (0.442761 --> 0.442193).  Saving model ...\n",
      "Validation loss decreased (0.442193 --> 0.441635).  Saving model ...\n",
      "Validation loss decreased (0.441635 --> 0.441079).  Saving model ...\n",
      "Validation loss decreased (0.441079 --> 0.440544).  Saving model ...\n",
      "Validation loss decreased (0.440544 --> 0.439997).  Saving model ...\n",
      "Validation loss decreased (0.439997 --> 0.439474).  Saving model ...\n",
      "Validation loss decreased (0.439474 --> 0.438951).  Saving model ...\n",
      "Validation loss decreased (0.438951 --> 0.438442).  Saving model ...\n",
      "Validation loss decreased (0.438442 --> 0.437932).  Saving model ...\n",
      "Validation loss decreased (0.437932 --> 0.437434).  Saving model ...\n",
      "Validation loss decreased (0.437434 --> 0.436937).  Saving model ...\n",
      "Validation loss decreased (0.436937 --> 0.436455).  Saving model ...\n",
      "Validation loss decreased (0.436455 --> 0.435971).  Saving model ...\n",
      "Validation loss decreased (0.435971 --> 0.435502).  Saving model ...\n",
      "Validation loss decreased (0.435502 --> 0.435031).  Saving model ...\n",
      "Validation loss decreased (0.435031 --> 0.434574).  Saving model ...\n",
      "Validation loss decreased (0.434574 --> 0.434117).  Saving model ...\n",
      "Validation loss decreased (0.434117 --> 0.433671).  Saving model ...\n",
      "Validation loss decreased (0.433671 --> 0.433224).  Saving model ...\n",
      "Validation loss decreased (0.433224 --> 0.432791).  Saving model ...\n",
      "Validation loss decreased (0.432791 --> 0.432350).  Saving model ...\n",
      "Validation loss decreased (0.432350 --> 0.431933).  Saving model ...\n",
      "Validation loss decreased (0.431933 --> 0.431504).  Saving model ...\n",
      "Validation loss decreased (0.431504 --> 0.431097).  Saving model ...\n",
      "Validation loss decreased (0.431097 --> 0.430676).  Saving model ...\n",
      "Validation loss decreased (0.430676 --> 0.430276).  Saving model ...\n",
      "Validation loss decreased (0.430276 --> 0.429868).  Saving model ...\n",
      "Validation loss decreased (0.429868 --> 0.429487).  Saving model ...\n",
      "Validation loss decreased (0.429487 --> 0.429076).  Saving model ...\n",
      "Validation loss decreased (0.429076 --> 0.428707).  Saving model ...\n",
      "Validation loss decreased (0.428707 --> 0.428319).  Saving model ...\n",
      "Validation loss decreased (0.428319 --> 0.427944).  Saving model ...\n",
      "Validation loss decreased (0.427944 --> 0.427571).  Saving model ...\n",
      "Validation loss decreased (0.427571 --> 0.427202).  Saving model ...\n",
      "Validation loss decreased (0.427202 --> 0.426836).  Saving model ...\n",
      "Validation loss decreased (0.426836 --> 0.426479).  Saving model ...\n",
      "Validation loss decreased (0.426479 --> 0.426126).  Saving model ...\n",
      "Validation loss decreased (0.426126 --> 0.425772).  Saving model ...\n",
      "Validation loss decreased (0.425772 --> 0.425429).  Saving model ...\n",
      "Validation loss decreased (0.425429 --> 0.425083).  Saving model ...\n",
      "Validation loss decreased (0.425083 --> 0.424746).  Saving model ...\n",
      "Validation loss decreased (0.424746 --> 0.424407).  Saving model ...\n",
      "Validation loss decreased (0.424407 --> 0.424081).  Saving model ...\n",
      "Validation loss decreased (0.424081 --> 0.423754).  Saving model ...\n",
      "Validation loss decreased (0.423754 --> 0.423428).  Saving model ...\n",
      "Validation loss decreased (0.423428 --> 0.423114).  Saving model ...\n",
      "Validation loss decreased (0.423114 --> 0.422793).  Saving model ...\n",
      "Validation loss decreased (0.422793 --> 0.422482).  Saving model ...\n",
      "Validation loss decreased (0.422482 --> 0.422177).  Saving model ...\n",
      "Validation loss decreased (0.422177 --> 0.421868).  Saving model ...\n",
      "Validation loss decreased (0.421868 --> 0.421565).  Saving model ...\n",
      "Validation loss decreased (0.421565 --> 0.421266).  Saving model ...\n",
      "Validation loss decreased (0.421266 --> 0.420971).  Saving model ...\n",
      "Validation loss decreased (0.420971 --> 0.420678).  Saving model ...\n",
      "Validation loss decreased (0.420678 --> 0.420383).  Saving model ...\n",
      "Validation loss decreased (0.420383 --> 0.420100).  Saving model ...\n",
      "Validation loss decreased (0.420100 --> 0.419819).  Saving model ...\n",
      "Validation loss decreased (0.419819 --> 0.419536).  Saving model ...\n",
      "Validation loss decreased (0.419536 --> 0.419260).  Saving model ...\n",
      "Validation loss decreased (0.419260 --> 0.418983).  Saving model ...\n",
      "Validation loss decreased (0.418983 --> 0.418717).  Saving model ...\n",
      "Validation loss decreased (0.418717 --> 0.418439).  Saving model ...\n",
      "Validation loss decreased (0.418439 --> 0.418191).  Saving model ...\n",
      "Validation loss decreased (0.418191 --> 0.417909).  Saving model ...\n",
      "Validation loss decreased (0.417909 --> 0.417669).  Saving model ...\n",
      "Validation loss decreased (0.417669 --> 0.417394).  Saving model ...\n",
      "Validation loss decreased (0.417394 --> 0.417158).  Saving model ...\n",
      "Validation loss decreased (0.417158 --> 0.416885).  Saving model ...\n",
      "Validation loss decreased (0.416885 --> 0.416657).  Saving model ...\n",
      "Validation loss decreased (0.416657 --> 0.416387).  Saving model ...\n",
      "Validation loss decreased (0.416387 --> 0.416162).  Saving model ...\n",
      "Validation loss decreased (0.416162 --> 0.415917).  Saving model ...\n",
      "Validation loss decreased (0.415917 --> 0.415676).  Saving model ...\n",
      "Validation loss decreased (0.415676 --> 0.415448).  Saving model ...\n",
      "Validation loss decreased (0.415448 --> 0.415210).  Saving model ...\n",
      "Validation loss decreased (0.415210 --> 0.414988).  Saving model ...\n",
      "Validation loss decreased (0.414988 --> 0.414751).  Saving model ...\n",
      "Validation loss decreased (0.414751 --> 0.414536).  Saving model ...\n",
      "Validation loss decreased (0.414536 --> 0.414296).  Saving model ...\n",
      "Validation loss decreased (0.414296 --> 0.414092).  Saving model ...\n",
      "Validation loss decreased (0.414092 --> 0.413866).  Saving model ...\n",
      "Validation loss decreased (0.413866 --> 0.413650).  Saving model ...\n",
      "Validation loss decreased (0.413650 --> 0.413427).  Saving model ...\n",
      "Validation loss decreased (0.413427 --> 0.413225).  Saving model ...\n",
      "Validation loss decreased (0.413225 --> 0.413003).  Saving model ...\n",
      "Validation loss decreased (0.413003 --> 0.412802).  Saving model ...\n",
      "Validation loss decreased (0.412802 --> 0.412578).  Saving model ...\n",
      "Validation loss decreased (0.412578 --> 0.412387).  Saving model ...\n",
      "Validation loss decreased (0.412387 --> 0.412175).  Saving model ...\n",
      "Validation loss decreased (0.412175 --> 0.411970).  Saving model ...\n",
      "Validation loss decreased (0.411970 --> 0.411772).  Saving model ...\n",
      "Validation loss decreased (0.411772 --> 0.411576).  Saving model ...\n",
      "Validation loss decreased (0.411576 --> 0.411371).  Saving model ...\n",
      "Validation loss decreased (0.411371 --> 0.411177).  Saving model ...\n",
      "Validation loss decreased (0.411177 --> 0.410988).  Saving model ...\n",
      "Validation loss decreased (0.410988 --> 0.410790).  Saving model ...\n",
      "Validation loss decreased (0.410790 --> 0.410606).  Saving model ...\n",
      "Validation loss decreased (0.410606 --> 0.410411).  Saving model ...\n",
      "Validation loss decreased (0.410411 --> 0.410225).  Saving model ...\n",
      "Validation loss decreased (0.410225 --> 0.410038).  Saving model ...\n",
      "Validation loss decreased (0.410038 --> 0.409858).  Saving model ...\n",
      "Validation loss decreased (0.409858 --> 0.409674).  Saving model ...\n",
      "Validation loss decreased (0.409674 --> 0.409489).  Saving model ...\n",
      "Validation loss decreased (0.409489 --> 0.409308).  Saving model ...\n",
      "Validation loss decreased (0.409308 --> 0.409139).  Saving model ...\n",
      "Validation loss decreased (0.409139 --> 0.408952).  Saving model ...\n",
      "Validation loss decreased (0.408952 --> 0.408786).  Saving model ...\n",
      "Validation loss decreased (0.408786 --> 0.408603).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.408603 --> 0.408435).  Saving model ...\n",
      "Validation loss decreased (0.408435 --> 0.408257).  Saving model ...\n",
      "Validation loss decreased (0.408257 --> 0.408105).  Saving model ...\n",
      "Validation loss decreased (0.408105 --> 0.407910).  Saving model ...\n",
      "Validation loss decreased (0.407910 --> 0.407765).  Saving model ...\n",
      "Validation loss decreased (0.407765 --> 0.407584).  Saving model ...\n",
      "Validation loss decreased (0.407584 --> 0.407432).  Saving model ...\n",
      "Validation loss decreased (0.407432 --> 0.407260).  Saving model ...\n",
      "Validation loss decreased (0.407260 --> 0.407098).  Saving model ...\n",
      "Validation loss decreased (0.407098 --> 0.406951).  Saving model ...\n",
      "Validation loss decreased (0.406951 --> 0.406771).  Saving model ...\n",
      "Validation loss decreased (0.406771 --> 0.406640).  Saving model ...\n",
      "Validation loss decreased (0.406640 --> 0.406456).  Saving model ...\n",
      "Validation loss decreased (0.406456 --> 0.406321).  Saving model ...\n",
      "Validation loss decreased (0.406321 --> 0.406158).  Saving model ...\n",
      "Validation loss decreased (0.406158 --> 0.406010).  Saving model ...\n",
      "Validation loss decreased (0.406010 --> 0.405856).  Saving model ...\n",
      "Validation loss decreased (0.405856 --> 0.405703).  Saving model ...\n",
      "Validation loss decreased (0.405703 --> 0.405553).  Saving model ...\n",
      "Validation loss decreased (0.405553 --> 0.405416).  Saving model ...\n",
      "Validation loss decreased (0.405416 --> 0.405253).  Saving model ...\n",
      "Validation loss decreased (0.405253 --> 0.405122).  Saving model ...\n",
      "Validation loss decreased (0.405122 --> 0.404973).  Saving model ...\n",
      "Validation loss decreased (0.404973 --> 0.404827).  Saving model ...\n",
      "Validation loss decreased (0.404827 --> 0.404699).  Saving model ...\n",
      "Validation loss decreased (0.404699 --> 0.404538).  Saving model ...\n",
      "Validation loss decreased (0.404538 --> 0.404418).  Saving model ...\n",
      "Validation loss decreased (0.404418 --> 0.404262).  Saving model ...\n",
      "Validation loss decreased (0.404262 --> 0.404134).  Saving model ...\n",
      "Validation loss decreased (0.404134 --> 0.404004).  Saving model ...\n",
      "Validation loss decreased (0.404004 --> 0.403856).  Saving model ...\n",
      "Validation loss decreased (0.403856 --> 0.403738).  Saving model ...\n",
      "Validation loss decreased (0.403738 --> 0.403603).  Saving model ...\n",
      "Validation loss decreased (0.403603 --> 0.403464).  Saving model ...\n",
      "Validation loss decreased (0.403464 --> 0.403349).  Saving model ...\n",
      "Validation loss decreased (0.403349 --> 0.403205).  Saving model ...\n",
      "Validation loss decreased (0.403205 --> 0.403089).  Saving model ...\n",
      "Validation loss decreased (0.403089 --> 0.402955).  Saving model ...\n",
      "Validation loss decreased (0.402955 --> 0.402840).  Saving model ...\n",
      "Validation loss decreased (0.402840 --> 0.402701).  Saving model ...\n",
      "Validation loss decreased (0.402701 --> 0.402595).  Saving model ...\n",
      "Validation loss decreased (0.402595 --> 0.402464).  Saving model ...\n",
      "Validation loss decreased (0.402464 --> 0.402350).  Saving model ...\n",
      "Validation loss decreased (0.402350 --> 0.402218).  Saving model ...\n",
      "Validation loss decreased (0.402218 --> 0.402117).  Saving model ...\n",
      "Validation loss decreased (0.402117 --> 0.401974).  Saving model ...\n",
      "Validation loss decreased (0.401974 --> 0.401881).  Saving model ...\n",
      "Validation loss decreased (0.401881 --> 0.401734).  Saving model ...\n",
      "Validation loss decreased (0.401734 --> 0.401656).  Saving model ...\n",
      "Validation loss decreased (0.401656 --> 0.401508).  Saving model ...\n",
      "Validation loss decreased (0.401508 --> 0.401413).  Saving model ...\n",
      "Validation loss decreased (0.401413 --> 0.401285).  Saving model ...\n",
      "Validation loss decreased (0.401285 --> 0.401180).  Saving model ...\n",
      "Validation loss decreased (0.401180 --> 0.401064).  Saving model ...\n",
      "Validation loss decreased (0.401064 --> 0.400942).  Saving model ...\n",
      "Validation loss decreased (0.400942 --> 0.400847).  Saving model ...\n",
      "Validation loss decreased (0.400847 --> 0.400731).  Saving model ...\n",
      "Validation loss decreased (0.400731 --> 0.400618).  Saving model ...\n",
      "Validation loss decreased (0.400618 --> 0.400511).  Saving model ...\n",
      "Validation loss decreased (0.400511 --> 0.400403).  Saving model ...\n",
      "Validation loss decreased (0.400403 --> 0.400291).  Saving model ...\n",
      "Validation loss decreased (0.400291 --> 0.400189).  Saving model ...\n",
      "Validation loss decreased (0.400189 --> 0.400076).  Saving model ...\n",
      "Validation loss decreased (0.400076 --> 0.399970).  Saving model ...\n",
      "Validation loss decreased (0.399970 --> 0.399873).  Saving model ...\n",
      "Validation loss decreased (0.399873 --> 0.399760).  Saving model ...\n",
      "Validation loss decreased (0.399760 --> 0.399658).  Saving model ...\n",
      "Validation loss decreased (0.399658 --> 0.399554).  Saving model ...\n",
      "Validation loss decreased (0.399554 --> 0.399458).  Saving model ...\n",
      "Validation loss decreased (0.399458 --> 0.399347).  Saving model ...\n",
      "Validation loss decreased (0.399347 --> 0.399259).  Saving model ...\n",
      "Validation loss decreased (0.399259 --> 0.399149).  Saving model ...\n",
      "Validation loss decreased (0.399149 --> 0.399054).  Saving model ...\n",
      "Validation loss decreased (0.399054 --> 0.398951).  Saving model ...\n",
      "Validation loss decreased (0.398951 --> 0.398862).  Saving model ...\n",
      "Validation loss decreased (0.398862 --> 0.398762).  Saving model ...\n",
      "Validation loss decreased (0.398762 --> 0.398652).  Saving model ...\n",
      "Validation loss decreased (0.398652 --> 0.398580).  Saving model ...\n",
      "Validation loss decreased (0.398580 --> 0.398462).  Saving model ...\n",
      "Validation loss decreased (0.398462 --> 0.398393).  Saving model ...\n",
      "Validation loss decreased (0.398393 --> 0.398264).  Saving model ...\n",
      "Validation loss decreased (0.398264 --> 0.398210).  Saving model ...\n",
      "Validation loss decreased (0.398210 --> 0.398089).  Saving model ...\n",
      "Validation loss decreased (0.398089 --> 0.398011).  Saving model ...\n",
      "Validation loss decreased (0.398011 --> 0.397906).  Saving model ...\n",
      "Validation loss decreased (0.397906 --> 0.397834).  Saving model ...\n",
      "Validation loss decreased (0.397834 --> 0.397718).  Saving model ...\n",
      "Validation loss decreased (0.397718 --> 0.397664).  Saving model ...\n",
      "Validation loss decreased (0.397664 --> 0.397537).  Saving model ...\n",
      "Validation loss decreased (0.397537 --> 0.397478).  Saving model ...\n",
      "Validation loss decreased (0.397478 --> 0.397370).  Saving model ...\n",
      "Validation loss decreased (0.397370 --> 0.397298).  Saving model ...\n",
      "Validation loss decreased (0.397298 --> 0.397201).  Saving model ...\n",
      "Validation loss decreased (0.397201 --> 0.397115).  Saving model ...\n",
      "Validation loss decreased (0.397115 --> 0.397037).  Saving model ...\n",
      "Validation loss decreased (0.397037 --> 0.396935).  Saving model ...\n",
      "Validation loss decreased (0.396935 --> 0.396868).  Saving model ...\n",
      "Validation loss decreased (0.396868 --> 0.396763).  Saving model ...\n",
      "Validation loss decreased (0.396763 --> 0.396696).  Saving model ...\n",
      "Validation loss decreased (0.396696 --> 0.396603).  Saving model ...\n",
      "Validation loss decreased (0.396603 --> 0.396519).  Saving model ...\n",
      "Validation loss decreased (0.396519 --> 0.396444).  Saving model ...\n",
      "Validation loss decreased (0.396444 --> 0.396352).  Saving model ...\n",
      "Validation loss decreased (0.396352 --> 0.396271).  Saving model ...\n",
      "Validation loss decreased (0.396271 --> 0.396183).  Saving model ...\n",
      "Validation loss decreased (0.396183 --> 0.396109).  Saving model ...\n",
      "Validation loss decreased (0.396109 --> 0.396025).  Saving model ...\n",
      "Validation loss decreased (0.396025 --> 0.395943).  Saving model ...\n",
      "Validation loss decreased (0.395943 --> 0.395848).  Saving model ...\n",
      "Validation loss decreased (0.395848 --> 0.395791).  Saving model ...\n",
      "Validation loss decreased (0.395791 --> 0.395693).  Saving model ...\n",
      "Validation loss decreased (0.395693 --> 0.395621).  Saving model ...\n",
      "Validation loss decreased (0.395621 --> 0.395547).  Saving model ...\n",
      "Validation loss decreased (0.395547 --> 0.395453).  Saving model ...\n",
      "Validation loss decreased (0.395453 --> 0.395388).  Saving model ...\n",
      "Validation loss decreased (0.395388 --> 0.395302).  Saving model ...\n",
      "Validation loss decreased (0.395302 --> 0.395233).  Saving model ...\n",
      "Validation loss decreased (0.395233 --> 0.395153).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.395153 --> 0.395070).  Saving model ...\n",
      "Validation loss decreased (0.395070 --> 0.395005).  Saving model ...\n",
      "Validation loss decreased (0.395005 --> 0.394917).  Saving model ...\n",
      "Validation loss decreased (0.394917 --> 0.394858).  Saving model ...\n",
      "Validation loss decreased (0.394858 --> 0.394767).  Saving model ...\n",
      "Validation loss decreased (0.394767 --> 0.394704).  Saving model ...\n",
      "Validation loss decreased (0.394704 --> 0.394618).  Saving model ...\n",
      "Validation loss decreased (0.394618 --> 0.394569).  Saving model ...\n",
      "Validation loss decreased (0.394569 --> 0.394465).  Saving model ...\n",
      "Validation loss decreased (0.394465 --> 0.394429).  Saving model ...\n",
      "Validation loss decreased (0.394429 --> 0.394332).  Saving model ...\n",
      "Validation loss decreased (0.394332 --> 0.394265).  Saving model ...\n",
      "Validation loss decreased (0.394265 --> 0.394207).  Saving model ...\n",
      "Validation loss decreased (0.394207 --> 0.394114).  Saving model ...\n",
      "Validation loss decreased (0.394114 --> 0.394069).  Saving model ...\n",
      "Validation loss decreased (0.394069 --> 0.393981).  Saving model ...\n",
      "Validation loss decreased (0.393981 --> 0.393920).  Saving model ...\n",
      "Validation loss decreased (0.393920 --> 0.393842).  Saving model ...\n",
      "Validation loss decreased (0.393842 --> 0.393789).  Saving model ...\n",
      "Validation loss decreased (0.393789 --> 0.393695).  Saving model ...\n",
      "Validation loss decreased (0.393695 --> 0.393643).  Saving model ...\n",
      "Validation loss decreased (0.393643 --> 0.393583).  Saving model ...\n",
      "Validation loss decreased (0.393583 --> 0.393487).  Saving model ...\n",
      "Validation loss decreased (0.393487 --> 0.393451).  Saving model ...\n",
      "Validation loss decreased (0.393451 --> 0.393367).  Saving model ...\n",
      "Validation loss decreased (0.393367 --> 0.393311).  Saving model ...\n",
      "Validation loss decreased (0.393311 --> 0.393238).  Saving model ...\n",
      "Validation loss decreased (0.393238 --> 0.393168).  Saving model ...\n",
      "Validation loss decreased (0.393168 --> 0.393119).  Saving model ...\n",
      "Validation loss decreased (0.393119 --> 0.393035).  Saving model ...\n",
      "Validation loss decreased (0.393035 --> 0.392979).  Saving model ...\n",
      "Validation loss decreased (0.392979 --> 0.392918).  Saving model ...\n",
      "Validation loss decreased (0.392918 --> 0.392848).  Saving model ...\n",
      "Validation loss decreased (0.392848 --> 0.392795).  Saving model ...\n",
      "Validation loss decreased (0.392795 --> 0.392722).  Saving model ...\n",
      "Validation loss decreased (0.392722 --> 0.392655).  Saving model ...\n",
      "Validation loss decreased (0.392655 --> 0.392607).  Saving model ...\n",
      "Validation loss decreased (0.392607 --> 0.392521).  Saving model ...\n",
      "Validation loss decreased (0.392521 --> 0.392485).  Saving model ...\n",
      "Validation loss decreased (0.392485 --> 0.392400).  Saving model ...\n",
      "Validation loss decreased (0.392400 --> 0.392353).  Saving model ...\n",
      "Validation loss decreased (0.392353 --> 0.392278).  Saving model ...\n",
      "Validation loss decreased (0.392278 --> 0.392234).  Saving model ...\n",
      "Validation loss decreased (0.392234 --> 0.392155).  Saving model ...\n",
      "Validation loss decreased (0.392155 --> 0.392101).  Saving model ...\n",
      "Validation loss decreased (0.392101 --> 0.392040).  Saving model ...\n",
      "Validation loss decreased (0.392040 --> 0.391963).  Saving model ...\n",
      "Validation loss decreased (0.391963 --> 0.391926).  Saving model ...\n",
      "Validation loss decreased (0.391926 --> 0.391845).  Saving model ...\n",
      "Validation loss decreased (0.391845 --> 0.391803).  Saving model ...\n",
      "Validation loss decreased (0.391803 --> 0.391729).  Saving model ...\n",
      "Validation loss decreased (0.391729 --> 0.391682).  Saving model ...\n",
      "Validation loss decreased (0.391682 --> 0.391612).  Saving model ...\n",
      "Validation loss decreased (0.391612 --> 0.391562).  Saving model ...\n",
      "Validation loss decreased (0.391562 --> 0.391490).  Saving model ...\n",
      "Validation loss decreased (0.391490 --> 0.391452).  Saving model ...\n",
      "Validation loss decreased (0.391452 --> 0.391369).  Saving model ...\n",
      "Validation loss decreased (0.391369 --> 0.391332).  Saving model ...\n",
      "Validation loss decreased (0.391332 --> 0.391264).  Saving model ...\n",
      "Validation loss decreased (0.391264 --> 0.391217).  Saving model ...\n",
      "Validation loss decreased (0.391217 --> 0.391146).  Saving model ...\n",
      "Validation loss decreased (0.391146 --> 0.391112).  Saving model ...\n",
      "Validation loss decreased (0.391112 --> 0.391015).  Saving model ...\n",
      "Validation loss decreased (0.391015 --> 0.390999).  Saving model ...\n",
      "Validation loss decreased (0.390999 --> 0.390909).  Saving model ...\n",
      "Validation loss decreased (0.390909 --> 0.390890).  Saving model ...\n",
      "Validation loss decreased (0.390890 --> 0.390804).  Saving model ...\n",
      "Validation loss decreased (0.390804 --> 0.390771).  Saving model ...\n",
      "Validation loss decreased (0.390771 --> 0.390696).  Saving model ...\n",
      "Validation loss decreased (0.390696 --> 0.390667).  Saving model ...\n",
      "Validation loss decreased (0.390667 --> 0.390606).  Saving model ...\n",
      "Validation loss decreased (0.390606 --> 0.390544).  Saving model ...\n",
      "Validation loss decreased (0.390544 --> 0.390495).  Saving model ...\n",
      "Validation loss decreased (0.390495 --> 0.390440).  Saving model ...\n",
      "Validation loss decreased (0.390440 --> 0.390389).  Saving model ...\n",
      "Validation loss decreased (0.390389 --> 0.390334).  Saving model ...\n",
      "Validation loss decreased (0.390334 --> 0.390294).  Saving model ...\n",
      "Validation loss decreased (0.390294 --> 0.390228).  Saving model ...\n",
      "Validation loss decreased (0.390228 --> 0.390173).  Saving model ...\n",
      "Validation loss decreased (0.390173 --> 0.390143).  Saving model ...\n",
      "Validation loss decreased (0.390143 --> 0.390060).  Saving model ...\n",
      "Validation loss decreased (0.390060 --> 0.390041).  Saving model ...\n",
      "Validation loss decreased (0.390041 --> 0.389964).  Saving model ...\n",
      "Validation loss decreased (0.389964 --> 0.389933).  Saving model ...\n",
      "Validation loss decreased (0.389933 --> 0.389855).  Saving model ...\n",
      "Validation loss decreased (0.389855 --> 0.389833).  Saving model ...\n",
      "Validation loss decreased (0.389833 --> 0.389768).  Saving model ...\n",
      "Validation loss decreased (0.389768 --> 0.389718).  Saving model ...\n",
      "Validation loss decreased (0.389718 --> 0.389662).  Saving model ...\n",
      "Validation loss decreased (0.389662 --> 0.389625).  Saving model ...\n",
      "Validation loss decreased (0.389625 --> 0.389566).  Saving model ...\n",
      "Validation loss decreased (0.389566 --> 0.389507).  Saving model ...\n",
      "Validation loss decreased (0.389507 --> 0.389483).  Saving model ...\n",
      "Validation loss decreased (0.389483 --> 0.389408).  Saving model ...\n",
      "Validation loss decreased (0.389408 --> 0.389363).  Saving model ...\n",
      "Validation loss decreased (0.389363 --> 0.389321).  Saving model ...\n",
      "Validation loss decreased (0.389321 --> 0.389275).  Saving model ...\n",
      "Validation loss decreased (0.389275 --> 0.389205).  Saving model ...\n",
      "Validation loss decreased (0.389205 --> 0.389183).  Saving model ...\n",
      "Validation loss decreased (0.389183 --> 0.389108).  Saving model ...\n",
      "Validation loss decreased (0.389108 --> 0.389083).  Saving model ...\n",
      "Validation loss decreased (0.389083 --> 0.389008).  Saving model ...\n",
      "Validation loss decreased (0.389008 --> 0.388978).  Saving model ...\n",
      "Validation loss decreased (0.388978 --> 0.388915).  Saving model ...\n",
      "Validation loss decreased (0.388915 --> 0.388885).  Saving model ...\n",
      "Validation loss decreased (0.388885 --> 0.388804).  Saving model ...\n",
      "Validation loss decreased (0.388804 --> 0.388796).  Saving model ...\n",
      "Validation loss decreased (0.388796 --> 0.388719).  Saving model ...\n",
      "Validation loss decreased (0.388719 --> 0.388687).  Saving model ...\n",
      "Validation loss decreased (0.388687 --> 0.388626).  Saving model ...\n",
      "Validation loss decreased (0.388626 --> 0.388601).  Saving model ...\n",
      "Validation loss decreased (0.388601 --> 0.388529).  Saving model ...\n",
      "Validation loss decreased (0.388529 --> 0.388482).  Saving model ...\n",
      "Validation loss decreased (0.388482 --> 0.388450).  Saving model ...\n",
      "Validation loss decreased (0.388450 --> 0.388395).  Saving model ...\n",
      "Validation loss decreased (0.388395 --> 0.388349).  Saving model ...\n",
      "Validation loss decreased (0.388349 --> 0.388303).  Saving model ...\n",
      "Validation loss decreased (0.388303 --> 0.388248).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.388248 --> 0.388207).  Saving model ...\n",
      "Validation loss decreased (0.388207 --> 0.388166).  Saving model ...\n",
      "Validation loss decreased (0.388166 --> 0.388107).  Saving model ...\n",
      "Validation loss decreased (0.388107 --> 0.388076).  Saving model ...\n",
      "Validation loss decreased (0.388076 --> 0.388015).  Saving model ...\n",
      "Validation loss decreased (0.388015 --> 0.387986).  Saving model ...\n",
      "Validation loss decreased (0.387986 --> 0.387908).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.387908 --> 0.387824).  Saving model ...\n",
      "Validation loss decreased (0.387824 --> 0.387806).  Saving model ...\n",
      "Validation loss decreased (0.387806 --> 0.387742).  Saving model ...\n",
      "Validation loss decreased (0.387742 --> 0.387710).  Saving model ...\n",
      "Validation loss decreased (0.387710 --> 0.387654).  Saving model ...\n",
      "Validation loss decreased (0.387654 --> 0.387621).  Saving model ...\n",
      "Validation loss decreased (0.387621 --> 0.387564).  Saving model ...\n",
      "Validation loss decreased (0.387564 --> 0.387535).  Saving model ...\n",
      "Validation loss decreased (0.387535 --> 0.387473).  Saving model ...\n",
      "Validation loss decreased (0.387473 --> 0.387441).  Saving model ...\n",
      "Validation loss decreased (0.387441 --> 0.387400).  Saving model ...\n",
      "Validation loss decreased (0.387400 --> 0.387359).  Saving model ...\n",
      "Validation loss decreased (0.387359 --> 0.387296).  Saving model ...\n",
      "Validation loss decreased (0.387296 --> 0.387270).  Saving model ...\n",
      "Validation loss decreased (0.387270 --> 0.387223).  Saving model ...\n",
      "Validation loss decreased (0.387223 --> 0.387176).  Saving model ...\n",
      "Validation loss decreased (0.387176 --> 0.387143).  Saving model ...\n",
      "Validation loss decreased (0.387143 --> 0.387085).  Saving model ...\n",
      "Validation loss decreased (0.387085 --> 0.387056).  Saving model ...\n",
      "Validation loss decreased (0.387056 --> 0.387001).  Saving model ...\n",
      "Validation loss decreased (0.387001 --> 0.386977).  Saving model ...\n",
      "Validation loss decreased (0.386977 --> 0.386916).  Saving model ...\n",
      "Validation loss decreased (0.386916 --> 0.386874).  Saving model ...\n",
      "Validation loss decreased (0.386874 --> 0.386838).  Saving model ...\n",
      "Validation loss decreased (0.386838 --> 0.386802).  Saving model ...\n",
      "Validation loss decreased (0.386802 --> 0.386733).  Saving model ...\n",
      "Validation loss decreased (0.386733 --> 0.386729).  Saving model ...\n",
      "Validation loss decreased (0.386729 --> 0.386671).  Saving model ...\n",
      "Validation loss decreased (0.386671 --> 0.386605).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.386605 --> 0.386529).  Saving model ...\n",
      "Validation loss decreased (0.386529 --> 0.386519).  Saving model ...\n",
      "Validation loss decreased (0.386519 --> 0.386445).  Saving model ...\n",
      "Validation loss decreased (0.386445 --> 0.386435).  Saving model ...\n",
      "Validation loss decreased (0.386435 --> 0.386374).  Saving model ...\n",
      "Validation loss decreased (0.386374 --> 0.386336).  Saving model ...\n",
      "Validation loss decreased (0.386336 --> 0.386308).  Saving model ...\n",
      "Validation loss decreased (0.386308 --> 0.386256).  Saving model ...\n",
      "Validation loss decreased (0.386256 --> 0.386217).  Saving model ...\n",
      "Validation loss decreased (0.386217 --> 0.386187).  Saving model ...\n",
      "Validation loss decreased (0.386187 --> 0.386134).  Saving model ...\n",
      "Validation loss decreased (0.386134 --> 0.386106).  Saving model ...\n",
      "Validation loss decreased (0.386106 --> 0.386059).  Saving model ...\n",
      "Validation loss decreased (0.386059 --> 0.386013).  Saving model ...\n",
      "Validation loss decreased (0.386013 --> 0.385995).  Saving model ...\n",
      "Validation loss decreased (0.385995 --> 0.385932).  Saving model ...\n",
      "Validation loss decreased (0.385932 --> 0.385905).  Saving model ...\n",
      "Validation loss decreased (0.385905 --> 0.385868).  Saving model ...\n",
      "Validation loss decreased (0.385868 --> 0.385834).  Saving model ...\n",
      "Validation loss decreased (0.385834 --> 0.385770).  Saving model ...\n",
      "Validation loss decreased (0.385770 --> 0.385761).  Saving model ...\n",
      "Validation loss decreased (0.385761 --> 0.385700).  Saving model ...\n",
      "Validation loss decreased (0.385700 --> 0.385679).  Saving model ...\n",
      "Validation loss decreased (0.385679 --> 0.385625).  Saving model ...\n",
      "Validation loss decreased (0.385625 --> 0.385589).  Saving model ...\n",
      "Validation loss decreased (0.385589 --> 0.385556).  Saving model ...\n",
      "Validation loss decreased (0.385556 --> 0.385514).  Saving model ...\n",
      "Validation loss decreased (0.385514 --> 0.385474).  Saving model ...\n",
      "Validation loss decreased (0.385474 --> 0.385435).  Saving model ...\n",
      "Validation loss decreased (0.385435 --> 0.385411).  Saving model ...\n",
      "Validation loss decreased (0.385411 --> 0.385353).  Saving model ...\n",
      "Validation loss decreased (0.385353 --> 0.385335).  Saving model ...\n",
      "Validation loss decreased (0.385335 --> 0.385274).  Saving model ...\n",
      "Validation loss decreased (0.385274 --> 0.385249).  Saving model ...\n",
      "Validation loss decreased (0.385249 --> 0.385222).  Saving model ...\n",
      "Validation loss decreased (0.385222 --> 0.385167).  Saving model ...\n",
      "Validation loss decreased (0.385167 --> 0.385137).  Saving model ...\n",
      "Validation loss decreased (0.385137 --> 0.385104).  Saving model ...\n",
      "Validation loss decreased (0.385104 --> 0.385054).  Saving model ...\n",
      "Validation loss decreased (0.385054 --> 0.385029).  Saving model ...\n",
      "Validation loss decreased (0.385029 --> 0.384987).  Saving model ...\n",
      "Validation loss decreased (0.384987 --> 0.384945).  Saving model ...\n",
      "Validation loss decreased (0.384945 --> 0.384915).  Saving model ...\n",
      "Validation loss decreased (0.384915 --> 0.384890).  Saving model ...\n",
      "Validation loss decreased (0.384890 --> 0.384829).  Saving model ...\n",
      "Validation loss decreased (0.384829 --> 0.384815).  Saving model ...\n",
      "Validation loss decreased (0.384815 --> 0.384752).  Saving model ...\n",
      "Validation loss decreased (0.384752 --> 0.384733).  Saving model ...\n",
      "Validation loss decreased (0.384733 --> 0.384696).  Saving model ...\n",
      "Validation loss decreased (0.384696 --> 0.384659).  Saving model ...\n",
      "Validation loss decreased (0.384659 --> 0.384622).  Saving model ...\n",
      "Validation loss decreased (0.384622 --> 0.384582).  Saving model ...\n",
      "Validation loss decreased (0.384582 --> 0.384553).  Saving model ...\n",
      "Validation loss decreased (0.384553 --> 0.384515).  Saving model ...\n",
      "Validation loss decreased (0.384515 --> 0.384477).  Saving model ...\n",
      "Validation loss decreased (0.384477 --> 0.384436).  Saving model ...\n",
      "Validation loss decreased (0.384436 --> 0.384421).  Saving model ...\n",
      "Validation loss decreased (0.384421 --> 0.384352).  Saving model ...\n",
      "Validation loss decreased (0.384352 --> 0.384344).  Saving model ...\n",
      "Validation loss decreased (0.384344 --> 0.384294).  Saving model ...\n",
      "Validation loss decreased (0.384294 --> 0.384270).  Saving model ...\n",
      "Validation loss decreased (0.384270 --> 0.384234).  Saving model ...\n",
      "Validation loss decreased (0.384234 --> 0.384193).  Saving model ...\n",
      "Validation loss decreased (0.384193 --> 0.384164).  Saving model ...\n",
      "Validation loss decreased (0.384164 --> 0.384124).  Saving model ...\n",
      "Validation loss decreased (0.384124 --> 0.384103).  Saving model ...\n",
      "Validation loss decreased (0.384103 --> 0.384048).  Saving model ...\n",
      "Validation loss decreased (0.384048 --> 0.384040).  Saving model ...\n",
      "Validation loss decreased (0.384040 --> 0.383986).  Saving model ...\n",
      "Validation loss decreased (0.383986 --> 0.383964).  Saving model ...\n",
      "Validation loss decreased (0.383964 --> 0.383926).  Saving model ...\n",
      "Validation loss decreased (0.383926 --> 0.383882).  Saving model ...\n",
      "Validation loss decreased (0.383882 --> 0.383870).  Saving model ...\n",
      "Validation loss decreased (0.383870 --> 0.383819).  Saving model ...\n",
      "Validation loss decreased (0.383819 --> 0.383815).  Saving model ...\n",
      "Validation loss decreased (0.383815 --> 0.383731).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.383731 --> 0.383691).  Saving model ...\n",
      "Validation loss decreased (0.383691 --> 0.383670).  Saving model ...\n",
      "Validation loss decreased (0.383670 --> 0.383623).  Saving model ...\n",
      "Validation loss decreased (0.383623 --> 0.383607).  Saving model ...\n",
      "Validation loss decreased (0.383607 --> 0.383556).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.383556 --> 0.383535).  Saving model ...\n",
      "Validation loss decreased (0.383535 --> 0.383501).  Saving model ...\n",
      "Validation loss decreased (0.383501 --> 0.383474).  Saving model ...\n",
      "Validation loss decreased (0.383474 --> 0.383440).  Saving model ...\n",
      "Validation loss decreased (0.383440 --> 0.383394).  Saving model ...\n",
      "Validation loss decreased (0.383394 --> 0.383384).  Saving model ...\n",
      "Validation loss decreased (0.383384 --> 0.383332).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.383332 --> 0.383262).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.383262 --> 0.383196).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.383196 --> 0.383152).  Saving model ...\n",
      "Validation loss decreased (0.383152 --> 0.383140).  Saving model ...\n",
      "Validation loss decreased (0.383140 --> 0.383085).  Saving model ...\n",
      "Validation loss decreased (0.383085 --> 0.383078).  Saving model ...\n",
      "Validation loss decreased (0.383078 --> 0.383019).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.383019 --> 0.382957).  Saving model ...\n",
      "Validation loss decreased (0.382957 --> 0.382957).  Saving model ...\n",
      "Validation loss decreased (0.382957 --> 0.382895).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.382895 --> 0.382822).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.382822 --> 0.382743).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.382743 --> 0.382690).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.382690 --> 0.382637).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.382637 --> 0.382594).  Saving model ...\n",
      "Validation loss decreased (0.382594 --> 0.382574).  Saving model ...\n",
      "Validation loss decreased (0.382574 --> 0.382539).  Saving model ...\n",
      "Validation loss decreased (0.382539 --> 0.382523).  Saving model ...\n",
      "Validation loss decreased (0.382523 --> 0.382478).  Saving model ...\n",
      "Validation loss decreased (0.382478 --> 0.382448).  Saving model ...\n",
      "Validation loss decreased (0.382448 --> 0.382418).  Saving model ...\n",
      "Validation loss decreased (0.382418 --> 0.382399).  Saving model ...\n",
      "Validation loss decreased (0.382399 --> 0.382351).  Saving model ...\n",
      "Validation loss decreased (0.382351 --> 0.382343).  Saving model ...\n",
      "Validation loss decreased (0.382343 --> 0.382292).  Saving model ...\n",
      "Validation loss decreased (0.382292 --> 0.382278).  Saving model ...\n",
      "Validation loss decreased (0.382278 --> 0.382237).  Saving model ...\n",
      "Validation loss decreased (0.382237 --> 0.382225).  Saving model ...\n",
      "Validation loss decreased (0.382225 --> 0.382164).  Saving model ...\n",
      "Validation loss decreased (0.382164 --> 0.382159).  Saving model ...\n",
      "Validation loss decreased (0.382159 --> 0.382143).  Saving model ...\n",
      "Validation loss decreased (0.382143 --> 0.382068).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.382068 --> 0.382024).  Saving model ...\n",
      "Validation loss decreased (0.382024 --> 0.382024).  Saving model ...\n",
      "Validation loss decreased (0.382024 --> 0.381977).  Saving model ...\n",
      "Validation loss decreased (0.381977 --> 0.381958).  Saving model ...\n",
      "Validation loss decreased (0.381958 --> 0.381928).  Saving model ...\n",
      "Validation loss decreased (0.381928 --> 0.381890).  Saving model ...\n",
      "Validation loss decreased (0.381890 --> 0.381880).  Saving model ...\n",
      "Validation loss decreased (0.381880 --> 0.381839).  Saving model ...\n",
      "Validation loss decreased (0.381839 --> 0.381806).  Saving model ...\n",
      "Validation loss decreased (0.381806 --> 0.381794).  Saving model ...\n",
      "Validation loss decreased (0.381794 --> 0.381757).  Saving model ...\n",
      "Validation loss decreased (0.381757 --> 0.381720).  Saving model ...\n",
      "Validation loss decreased (0.381720 --> 0.381719).  Saving model ...\n",
      "Validation loss decreased (0.381719 --> 0.381660).  Saving model ...\n",
      "Validation loss decreased (0.381660 --> 0.381654).  Saving model ...\n",
      "Validation loss decreased (0.381654 --> 0.381596).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.381596 --> 0.381539).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.381539 --> 0.381494).  Saving model ...\n",
      "Validation loss decreased (0.381494 --> 0.381481).  Saving model ...\n",
      "Validation loss decreased (0.381481 --> 0.381450).  Saving model ...\n",
      "Validation loss decreased (0.381450 --> 0.381424).  Saving model ...\n",
      "Validation loss decreased (0.381424 --> 0.381385).  Saving model ...\n",
      "Validation loss decreased (0.381385 --> 0.381378).  Saving model ...\n",
      "Validation loss decreased (0.381378 --> 0.381333).  Saving model ...\n",
      "Validation loss decreased (0.381333 --> 0.381318).  Saving model ...\n",
      "Validation loss decreased (0.381318 --> 0.381285).  Saving model ...\n",
      "Validation loss decreased (0.381285 --> 0.381263).  Saving model ...\n",
      "Validation loss decreased (0.381263 --> 0.381225).  Saving model ...\n",
      "Validation loss decreased (0.381225 --> 0.381221).  Saving model ...\n",
      "Validation loss decreased (0.381221 --> 0.381160).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.381160 --> 0.381114).  Saving model ...\n",
      "Validation loss decreased (0.381114 --> 0.381111).  Saving model ...\n",
      "Validation loss decreased (0.381111 --> 0.381051).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.381051 --> 0.381014).  Saving model ...\n",
      "Validation loss decreased (0.381014 --> 0.381006).  Saving model ...\n",
      "Validation loss decreased (0.381006 --> 0.380947).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380947 --> 0.380880).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380880 --> 0.380818).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380818 --> 0.380786).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380786 --> 0.380738).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380738 --> 0.380697).  Saving model ...\n",
      "Validation loss decreased (0.380697 --> 0.380690).  Saving model ...\n",
      "Validation loss decreased (0.380690 --> 0.380661).  Saving model ...\n",
      "Validation loss decreased (0.380661 --> 0.380628).  Saving model ...\n",
      "Validation loss decreased (0.380628 --> 0.380626).  Saving model ...\n",
      "Validation loss decreased (0.380626 --> 0.380567).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380567 --> 0.380516).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380516 --> 0.380466).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380466 --> 0.380411).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380411 --> 0.380374).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380374 --> 0.380277).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380277 --> 0.380258).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380258 --> 0.380208).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380208 --> 0.380162).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380162 --> 0.380127).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380127 --> 0.380087).  Saving model ...\n",
      "Validation loss decreased (0.380087 --> 0.380082).  Saving model ...\n",
      "Validation loss decreased (0.380082 --> 0.380032).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.380032 --> 0.379969).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379969 --> 0.379921).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379921 --> 0.379873).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379873 --> 0.379828).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.379828 --> 0.379770).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379770 --> 0.379725).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379725 --> 0.379689).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379689 --> 0.379655).  Saving model ...\n",
      "Validation loss decreased (0.379655 --> 0.379630).  Saving model ...\n",
      "Validation loss decreased (0.379630 --> 0.379624).  Saving model ...\n",
      "Validation loss decreased (0.379624 --> 0.379575).  Saving model ...\n",
      "Validation loss decreased (0.379575 --> 0.379564).  Saving model ...\n",
      "Validation loss decreased (0.379564 --> 0.379533).  Saving model ...\n",
      "Validation loss decreased (0.379533 --> 0.379510).  Saving model ...\n",
      "Validation loss decreased (0.379510 --> 0.379493).  Saving model ...\n",
      "Validation loss decreased (0.379493 --> 0.379461).  Saving model ...\n",
      "Validation loss decreased (0.379461 --> 0.379447).  Saving model ...\n",
      "Validation loss decreased (0.379447 --> 0.379425).  Saving model ...\n",
      "Validation loss decreased (0.379425 --> 0.379381).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379381 --> 0.379313).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379313 --> 0.379280).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379280 --> 0.379228).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379228 --> 0.379194).  Saving model ...\n",
      "Validation loss decreased (0.379194 --> 0.379193).  Saving model ...\n",
      "Validation loss decreased (0.379193 --> 0.379138).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379138 --> 0.379105).  Saving model ...\n",
      "Validation loss decreased (0.379105 --> 0.379081).  Saving model ...\n",
      "Validation loss decreased (0.379081 --> 0.379071).  Saving model ...\n",
      "Validation loss decreased (0.379071 --> 0.379021).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.379021 --> 0.378941).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378941 --> 0.378910).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378910 --> 0.378866).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378866 --> 0.378828).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378828 --> 0.378783).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378783 --> 0.378734).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378734 --> 0.378710).  Saving model ...\n",
      "Validation loss decreased (0.378710 --> 0.378681).  Saving model ...\n",
      "Validation loss decreased (0.378681 --> 0.378673).  Saving model ...\n",
      "Validation loss decreased (0.378673 --> 0.378651).  Saving model ...\n",
      "Validation loss decreased (0.378651 --> 0.378618).  Saving model ...\n",
      "Validation loss decreased (0.378618 --> 0.378605).  Saving model ...\n",
      "Validation loss decreased (0.378605 --> 0.378572).  Saving model ...\n",
      "Validation loss decreased (0.378572 --> 0.378558).  Saving model ...\n",
      "Validation loss decreased (0.378558 --> 0.378542).  Saving model ...\n",
      "Validation loss decreased (0.378542 --> 0.378492).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378492 --> 0.378439).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378439 --> 0.378387).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378387 --> 0.378366).  Saving model ...\n",
      "Validation loss decreased (0.378366 --> 0.378359).  Saving model ...\n",
      "Validation loss decreased (0.378359 --> 0.378315).  Saving model ...\n",
      "Validation loss decreased (0.378315 --> 0.378302).  Saving model ...\n",
      "Validation loss decreased (0.378302 --> 0.378293).  Saving model ...\n",
      "Validation loss decreased (0.378293 --> 0.378261).  Saving model ...\n",
      "Validation loss decreased (0.378261 --> 0.378234).  Saving model ...\n",
      "Validation loss decreased (0.378234 --> 0.378207).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378207 --> 0.378134).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378134 --> 0.378094).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378094 --> 0.378064).  Saving model ...\n",
      "Validation loss decreased (0.378064 --> 0.378052).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.378052 --> 0.377980).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377980 --> 0.377955).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377955 --> 0.377906).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377906 --> 0.377873).  Saving model ...\n",
      "Validation loss decreased (0.377873 --> 0.377869).  Saving model ...\n",
      "Validation loss decreased (0.377869 --> 0.377839).  Saving model ...\n",
      "Validation loss decreased (0.377839 --> 0.377827).  Saving model ...\n",
      "Validation loss decreased (0.377827 --> 0.377791).  Saving model ...\n",
      "Validation loss decreased (0.377791 --> 0.377770).  Saving model ...\n",
      "Validation loss decreased (0.377770 --> 0.377761).  Saving model ...\n",
      "Validation loss decreased (0.377761 --> 0.377727).  Saving model ...\n",
      "Validation loss decreased (0.377727 --> 0.377718).  Saving model ...\n",
      "Validation loss decreased (0.377718 --> 0.377668).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377668 --> 0.377657).  Saving model ...\n",
      "Validation loss decreased (0.377657 --> 0.377601).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377601 --> 0.377558).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377558 --> 0.377500).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377500 --> 0.377455).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377455 --> 0.377411).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377411 --> 0.377393).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377393 --> 0.377351).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377351 --> 0.377327).  Saving model ...\n",
      "Validation loss decreased (0.377327 --> 0.377308).  Saving model ...\n",
      "Validation loss decreased (0.377308 --> 0.377302).  Saving model ...\n",
      "Validation loss decreased (0.377302 --> 0.377246).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377246 --> 0.377185).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377185 --> 0.377142).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377142 --> 0.377098).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377098 --> 0.377080).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.377080 --> 0.377053).  Saving model ...\n",
      "Validation loss decreased (0.377053 --> 0.377052).  Saving model ...\n",
      "Validation loss decreased (0.377052 --> 0.377011).  Saving model ...\n",
      "Validation loss decreased (0.377011 --> 0.377001).  Saving model ...\n",
      "Validation loss decreased (0.377001 --> 0.376990).  Saving model ...\n",
      "Validation loss decreased (0.376990 --> 0.376950).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376950 --> 0.376901).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376901 --> 0.376849).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376849 --> 0.376801).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376801 --> 0.376744).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376744 --> 0.376705).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376705 --> 0.376699).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376699 --> 0.376667).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376667 --> 0.376628).  Saving model ...\n",
      "Validation loss decreased (0.376628 --> 0.376613).  Saving model ...\n",
      "Validation loss decreased (0.376613 --> 0.376611).  Saving model ...\n",
      "Validation loss decreased (0.376611 --> 0.376557).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376557 --> 0.376506).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "EarlyStopping counter: 2 out of 20\n",
      "Validation loss decreased (0.376506 --> 0.376466).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Validation loss decreased (0.376466 --> 0.376428).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 20\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "net2 = Net()\n",
    "criterion2 = nn.CrossEntropyLoss()\n",
    "optimizer2 = optim.SGD(net2.parameters(), lr = 1e-1, weight_decay=1e-2)\n",
    "loss_scores2, validate_loss2 = fit(net2, criterion2, optimizer2, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
